# -*- coding: utf-8 -*-

import sys
import re

from urlparse import urljoin
from scrapy import Request, FormRequest
from scrapy.spider import Spider
from bs4 import BeautifulSoup
from realEstateCommittee.items import RealestatecommitteeItem
from realEstateCommittee.commons import _REPLACE_NBSP

reload(sys)
sys.setdefaultencoding('utf8')


class RealEstateCommittee_HC_YY_Spider(Spider):
    name = 'RealEstateCommittee_HC_YY'

    PROJECT = "project"
    PROJECT_URL = "project_url"
    PROJECT_DETAIL_URL = "project_detail_url"
    BUILDING_URL = "building_url"
    CLH = "clh"
    HOUSE_URL = "house_url"
    HOUSE_DETAIL_URL = "house_detail_url"

    BELONG_CITY = "belong_city"  # : "所在区县",
    PROJECT_ADDR = "project_addr"  # : "项目地址",
    PRESALE_PERMIT = "presale_permit"  # :"预售证号码",
    BUILD_NO = "build_no"  # :"本期住宅楼号/幢号",
    RESIDENCE_SET = "residence_set"  # :"本期住宅总套数",
    RESIDENCE_AREA = "residence_area"  # :"本期住宅总面积",
    COMMERCIAL_AREA = "commercial_area"  # :"本期商业总面积",
    SOLD_RESIDENCE_SET = "sold_residence_set"  # :"本期已售住宅总套数",
    UNSOLD_RESIDENCE_SET = "unsold_residence_set"  # :"本期未售住宅总套数",
    PROPERTY_TYPE = "property_type"  # :"物业类型/房屋类型",
    BUILDING_TYPE = "building_type"  # :"建筑类型",
    OPEN_TIME = "open_time"  # : "开盘时间",
    COMPLETE_TIME = "complete_time"  # : "竣工时间",
    DELIVERY_TIME = "delivery_time"  # : "交房时间",
    GREEN_RATIO = "green_ratio"  # :"绿化率",
    DEVELOPER = "developer"  # :"开发商名称",
    PROPERTY_COMPANY = "property_company"  # :"物业公司",
    REFER_PRICE = "refer_price"  # :"参考价",
    BUILDING_NO = "building_no"  # :"楼号/幢号",
    DECORATE_TYPE = "decorate_type"  # :"装修类型",
    FLOOR = "floor"  # :"楼层",
    HOUSE_NO = "house_no"  # :"房号",
    HOUSE_TYPE = "house_type"  # :"房型",
    HOUSE_AREA = "house_area"  # :"房屋建筑面积",
    INNER_AREA = "inner_area"  # :"套内面积",
    SALE_STATUS = "sale_status"  # :"销售状态"

    key_list = [BELONG_CITY, PROJECT_ADDR, PRESALE_PERMIT, BUILD_NO, RESIDENCE_SET, RESIDENCE_AREA, COMMERCIAL_AREA,
                SOLD_RESIDENCE_SET, UNSOLD_RESIDENCE_SET, PROPERTY_TYPE, BUILDING_TYPE, OPEN_TIME, COMPLETE_TIME,
                DELIVERY_TIME, GREEN_RATIO, DEVELOPER, PROPERTY_COMPANY, REFER_PRICE, BUILDING_NO, DECORATE_TYPE, FLOOR,
                HOUSE_NO, HOUSE_TYPE, HOUSE_AREA, INNER_AREA, SALE_STATUS]

    start_urls = ["http://xx.yyfdcw.com/NewHouse/BuildingList.aspx"]

    def make_requests_from_url(self, url):
        meta = dict()
        meta['url'] = url
        meta['page'] = 1
        return Request(url=url, callback=self.parse, meta=meta, dont_filter=True)

    def parse(self, response):
        """

        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        for project_tag in bs_obj.find_all("div", class_="listinfo"):
            meta[self.PROJECT] = _REPLACE_NBSP(project_tag.h3.a.get_text())
            meta[self.PROJECT_URL] = urljoin(response.url, project_tag.h3.a.attrs['href'])
            meta[self.PROJECT_DETAIL_URL] = meta[self.PROJECT_URL].replace("NewHouse.aspx", "DetailInfo.aspx")
            self.logger.info("项目名称:%s, 项目链接:%s, 项目详情页链接:%s" % (
                meta[self.PROJECT], meta[self.PROJECT_URL], meta[self.PROJECT_DETAIL_URL]))
            yield Request(meta[self.PROJECT_DETAIL_URL], callback=self.parse_project_detail, meta=meta,
                          dont_filter=True)

        meta.pop(self.PROJECT)
        next_button = bs_obj.find("a", attrs={"href": True, "disabled": None}, text={re.compile(u"下一页")})
        next_button = None
        if next_button:
            meta["page"] = meta["page"] + 1
            __VIEWSTATEGENERATOR = bs_obj.find("input", id="__VIEWSTATEGENERATOR").attrs["value"]
            __VIEWSTATE = bs_obj.find("input", id="__VIEWSTATE").attrs["value"]
            __EVENTARGUMENT = re.findall("'(.*?)'", next_button.attrs['href'])[1]
            __EVENTTARGET = "Pager"
            ddlYear = "-1"
            ddlArea = "-1"
            ddlZhuangtai = "-1"
            ddlPrice = "-1"
            ddlWyType = "-1"
            txtKeyWord = ""
            Pager_input = "1"
            form_data = dict({
                "__VIEWSTATEGENERATOR": __VIEWSTATEGENERATOR,
                "__VIEWSTATE": __VIEWSTATE,
                "__EVENTARGUMENT": __EVENTARGUMENT,
                "__EVENTTARGET": __EVENTTARGET,
                "ddlYear": ddlYear,
                "ddlArea": ddlArea,
                "ddlZhuangtai": ddlZhuangtai,
                "ddlPrice": ddlPrice,
                "ddlWyType": ddlWyType,
                "txtKeyWord": txtKeyWord,
                "Pager_input": Pager_input
            })
            self.logger.info("存在下一页, 页码为:%s" % __EVENTARGUMENT)
            yield FormRequest(response.url, formdata=form_data, meta=meta, dont_filter=True)
        else:
            self.logger.info("已到达最后一页, 页码为:%s" % meta["page"])

            pass

    def parse_project_detail(self, response):
        """
        DECORATE_TYPE = "decorate_type" # :"装修类型",
        BELONG_CITY = "belong_city" # : "所在区县",
        PROJECT_ADDR = "project_addr"   # : "项目地址",
        PROPERTY_TYPE = "property_type" # :"物业类型/房屋类型",
        BUILDING_TYPE = "building_type" # :"建筑类型",
        DEVELOPER = "developer" # :"开发商名称",
        GREEN_RATIO = "green_ratio" # :"绿化率",
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        meta[self.DECORATE_TYPE] = _REPLACE_NBSP(bs_obj.find("span", id="lblStandard").get_text())
        self.logger.info("项目名称:%s, 装修类型:%s" % (meta[self.PROJECT], meta[self.DECORATE_TYPE]))
        meta[self.BELONG_CITY] = bs_obj.find("span", id="lblArea").get_text()
        meta[self.PROJECT_ADDR] = bs_obj.find("b", text={re.compile(u"项目位置")}).parent.get_text().replace("项目位置：", "")
        meta[self.PROPERTY_TYPE] = bs_obj.find("span", id="lblHouseType").get_text()
        meta[self.BUILDING_TYPE] = bs_obj.find("span", id="lblBuildingType").get_text()
        meta[self.DEVELOPER] = bs_obj.find("span", id="lblDevelopers").get_text()
        meta[self.GREEN_RATIO] = bs_obj.find("span", id="lblVolumeRate").get_text()
        self.logger.info("所在区县:%s, 项目地址:%s, 物业类型/房屋类型:%s, 建筑类型:%s, 开发商名称:%s, 绿化率:%s"
                         % (meta[self.BELONG_CITY], meta[self.PROJECT_ADDR], meta[self.PROPERTY_TYPE],
                            meta[self.BUILDING_TYPE], meta[self.DEVELOPER], meta[self.GREEN_RATIO]))
        yield Request(meta[self.PROJECT_URL], callback=self.parse_project_index, meta=meta, dont_filter=True)
        pass

    def parse_project_index(self, response):
        """
        PROPERTY_COMPANY = "property_company"   # :"物业公司",
        BUILD_NO = "build_no"   # :"本期住宅楼号/幢号",
        BUILDING_NO = "building_no" # :"楼号/幢号",
        PRESALE_PERMIT = "presale_permit"   # :"预售证号码",
        RESIDENCE_SET = "residence_set" # :"本期住宅总套数",
        SOLD_RESIDENCE_SET = "sold_residence_set"   # :"本期已售住宅总套数",
        UNSOLD_RESIDENCE_SET = "unsold_residence_set"   # :"本期未售住宅总套数",
        OPEN_TIME = "open_time" # : "开盘时间",
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")

        meta[self.PROPERTY_COMPANY] = bs_obj.find("span", id="lblWgdw").get_text()
        project_infos = [item.get_text() for item in
                         bs_obj.find("td", text={re.compile(u"项目总套数")}).parent.find_next_sibling("tr").find_all("td")]
        meta[self.RESIDENCE_SET] = project_infos[0]
        meta[self.SOLD_RESIDENCE_SET] = project_infos[3]
        meta[self.UNSOLD_RESIDENCE_SET] = project_infos[1]
        self.logger.info("项目名称:%s, 物业公司:%s, 本期住宅总套数:%s, 本期已售住宅总套数:%s, 本期未售住宅总套数:%s" % (
            meta[self.PROJECT], meta[self.PROPERTY_COMPANY], meta[self.RESIDENCE_SET], meta[self.SOLD_RESIDENCE_SET],
            meta[self.UNSOLD_RESIDENCE_SET]))

        for building_tag in bs_obj.find("ul", class_="dongDate").find_all("li")[1:]:
            meta[self.BUILDING_NO] = building_tag.find("a").attrs["title"]
            meta[self.BUILD_NO] = building_tag.find("a").attrs["title"]
            meta[self.BUILDING_URL] = urljoin(response.url, building_tag.find("a").attrs["href"])
            meta[self.PRESALE_PERMIT] = building_tag.find_all("a")[1].get_text()
            meta[self.OPEN_TIME] = _REPLACE_NBSP(building_tag.find_all("td")[3].get_text())
            self.logger.info("项目名称:%s, 楼栋号:%s, 预售许可证号:%s, 开盘时间:%s, 楼栋链接:%s" % (
                meta[self.PROJECT], meta[self.BUILDING_NO], meta[self.PRESALE_PERMIT], meta[self.OPEN_TIME],
                meta[self.BUILDING_URL]))
            yield Request(meta[self.BUILDING_URL], callback=self.parse_building, meta=meta, dont_filter=True)

        pass

    def parse_building(self, response):
        """
        COMPLETE_TIME = "complete_time" # : "竣工时间",
        DELIVERY_TIME = "delivery_time" # : "交房时间",
        RESIDENCE_AREA = "residence_area"   # :"本期住宅总面积",
        COMMERCIAL_AREA = "commercial_area" # :"本期商业总面积",
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        meta[self.COMPLETE_TIME] = bs_obj.find("span", id="lblJgrq").get_text()
        meta[self.DELIVERY_TIME] = bs_obj.find("span", id="lblJfrq").get_text()
        meta[self.RESIDENCE_AREA] = bs_obj.find("span", id="lblZzmj").get_text()
        meta[self.COMMERCIAL_AREA] = bs_obj.find("span", id="lblSymj").get_text()
        # //*[@id="lblClh"]
        meta[self.CLH] = bs_obj.find("span", id="lblClh").get_text()
        self.logger.info("项目名称:%s, 楼栋号:%s, 竣工时间:%s, 交房时间:%s, 本期住宅总面积:%s, 本期商业总面积:%s, 测量号:%s" % (
            meta[self.PROJECT], meta[self.BUILDING_NO], meta[self.COMPLETE_TIME], meta[self.DELIVERY_TIME],
            meta[self.RESIDENCE_AREA], meta[self.COMMERCIAL_AREA], meta[self.CLH]))
        meta[self.HOUSE_URL] = "http://xx.yyfdcw.com/hetong/fdc_xxdxx.asp?id=%s" % meta[self.CLH]
        yield Request(meta[self.HOUSE_URL], callback=self.parse_house, meta=meta, dont_filter=True)
        pass

    def parse_house(self, response):
        """
        HOUSE_NO = "house_no"   # :"房号",
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        for input in bs_obj.find_all("input"):
            meta[self.HOUSE_NO] = input.attrs["value"]
            meta[self.HOUSE_DETAIL_URL] = urljoin(response.url, re.findall("'(.*?)'", input.attrs["onclick"])[0])
            self.logger.info("项目名称:%s, 楼栋号:%s, 预售许可证号:%s, 房号:%s, 房号链接:%s" % (
                meta[self.PROJECT], meta[self.BUILDING_NO], meta[self.PRESALE_PERMIT], meta[self.HOUSE_NO],
                meta[self.HOUSE_DETAIL_URL]))
            yield Request(meta[self.HOUSE_DETAIL_URL], callback=self.parse_house_detail, meta=meta, dont_filter=True)

        pass

    def parse_house_detail(self, response):
        """
        FLOOR = "floor" # :"楼层",
        REFER_PRICE = "refer_price" # :"参考价",
        HOUSE_TYPE = "house_type"   # :"房型",
        HOUSE_AREA = "house_area"   # :"房屋建筑面积",
        INNER_AREA = "inner_area"   # :"套内面积",
        SALE_STATUS = "sale_status" # :"销售状态"
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")

        def get_value(key):
            return _REPLACE_NBSP(bs_obj.find(text={re.compile(key)}).parent.find_next_sibling("td").get_text())

        meta[self.FLOOR] = get_value(u"名义层")
        meta[self.HOUSE_TYPE] = get_value(u"房型")
        meta[self.HOUSE_AREA] = get_value(u"预测建筑面积")
        meta[self.INNER_AREA] = get_value(u"预测套内面积")
        meta[self.SALE_STATUS] = get_value(u"状态")
        meta[self.REFER_PRICE] = get_value(u"申报价格")

        self.logger.info("项目名称:%s, 楼栋号:%s, 房号:%s, 楼层:%s, 参考价:%s, 房型:%s, 房屋建筑面积:%s, 套内面积:%s, 销售状态:%s"
                         % (meta[self.PROJECT], meta[self.BUILDING_NO], meta[self.HOUSE_NO], meta[self.FLOOR],
                            meta[self.REFER_PRICE], meta[self.HOUSE_TYPE], meta[self.HOUSE_AREA],
                            meta[self.INNER_AREA], meta[self.SALE_STATUS]))
        yield self.meta_to_item(meta)
        pass

    def meta_to_item(self, meta):
        item = RealestatecommitteeItem()
        for field in RealestatecommitteeItem.fields:
            if field not in meta.keys():
                item[field] = ""
            else:
                item[field] = meta[field]
        return item
